library(tidyverse)
── Attaching core tidyverse packages ──────────────────────────────────────────────────────────────────────────────────────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ ggplot2   3.5.2     ✔ tibble    3.3.0
✔ lubridate 1.9.4     ✔ tidyr     1.3.1
✔ purrr     1.1.0     
── Conflicts ────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the ]8;;http://conflicted.r-lib.org/conflicted package]8;; to force all conflicts to become errors
library(car)
G3;Loading required package: carData
gG3;
Attaching package: ‘car’

gG3;The following object is masked from ‘package:dplyr’:

    recode

gG3;The following object is masked from ‘package:purrr’:

    some

g
library(performance)
library(patchwork)

idea:

Sim violating data

Something different with the errors this time. Give students new illustration of how things can diverge from normality. Not asymmetrical, but maybe U-shaped errors? As in, mostly large in both directions, not usually small?

shapes <- .3
set.seed(1)
beta_errors <- rbeta(101, shape1 = shapes, shape2 = shapes) |>
  datawizard::standardize()
  # scale(scale = FALSE, center = TRUE)
plot(beta_errors)

hist(beta_errors)

set.seed(1)

error_inc_terms <- seq(.5, 3, length.out = 101)

df <- tibble(
  x = seq(-2, 2, length.out = 101),
  y_good = 2 + (-1.1 * x) + rnorm(101, 0, 1), 
  y_viol = 2 + (-1.1 * x) + beta_errors * error_inc_terms,
)

p_good <- df |>
  ggplot(aes(x=x, y=y_good)) +
  geom_point() +
  # geom_smooth(method = 'lm', se = FALSE) +
  NULL


p_beta <- df |>
  ggplot(aes(x=x, y=y_viol)) +
  geom_point() +
  # geom_smooth(method = 'lm', se = FALSE) +
  NULL

p_good + p_beta

Fit models

m_good <- lm(y_good ~ x, data = df)
m_viol <- lm(y_viol ~ x, data = df)
summary(m_viol)

Call:
lm(formula = y_viol ~ x, data = df)

Residuals:
    Min      1Q  Median      3Q     Max 
-4.0231 -1.3473  0.3146  1.5062  3.0814 

Coefficients:
            Estimate Std. Error t value Pr(>|t|)    
(Intercept)   2.1154     0.1846  11.459  < 2e-16 ***
x            -0.8130     0.1583  -5.136 1.41e-06 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 1.855 on 99 degrees of freedom
Multiple R-squared:  0.2104,    Adjusted R-squared:  0.2024 
F-statistic: 26.38 on 1 and 99 DF,  p-value: 1.408e-06

Plot diagnostics

check_model(m_viol)

Q-Q plot

plot(m_viol, which = 2)

Histogram of residuals

  • asymmetrical
  • fatter tails than we would expect
hist(m_viol$residuals)

It’s really interesting how the model can find a line such that the residuals look almost gaussian, even though the generative process was so far from gaussian.

Compare residuals to fitted values

plot(m_viol, which = 1)

car::Boot()

fit model

In reality, present this after showing the manual process.

Boot() draws R samples with the same number of observations as the original dataset.

set.seed(1)
m_viol_boot <- Boot(m_viol, R = 1000)
G3;Loading required namespace: boot
g
summary(m_viol_boot)

Number of bootstrap replications R = 1000 
            original    bootBias  bootSE  bootMed
(Intercept)  2.11540  0.00524979 0.18235  2.13003
x           -0.81301 -0.00071738 0.16164 -0.81038

The SE of the bootstrapped version is smaller – there is less variability in the bootstrapped data than in the original sample. Here’s the SE of the original sample:

summary(m_viol)$coefficients
              Estimate Std. Error   t value     Pr(>|t|)
(Intercept)  2.1153955  0.1846088 11.458803 7.444834e-20
x           -0.8130144  0.1583007 -5.135886 1.408443e-06
Confint(m_viol_boot, level = 0.95, type = 'perc')
Bootstrap percent confidence intervals

Consequently, the CIs are narrower for the bootstrapped model than they are for the original model. Original CIs:

Confint(m_viol, level = 0.95, type = 'perc')
              Estimate     2.5 %     97.5 %
(Intercept)  2.1153955  1.749092  2.4816994
x           -0.8130144 -1.127117 -0.4989114

plot diagnostics?

Doesn’t make a ton of sense because doesn’t really rely on same assumptions as the parametric version of the model.

plot(m_viol_boot)

Interesting, so apparently we want the t-values to be normally distributed.

check_model() throws an error.

# check_model(m_viol_boot)

fail: emmeans doesn’t work on boot object.

# m_viol_emm <- emmeans( ref_grid(m_viol, at = list(x = c(-2, -1, 0, 1, 2))), ~x)
# m_viol_emm |> as.data.frame()
# m_viol_boot_emm <- emmeans( ref_grid(m_viol_boot, at = list(x = c(-2, -1, 0, 1, 2))), ~x)

Manually bootstrap from this data

boot_sample_size <- 25
n_resamples <- 50

orig_data <- df |>
  select(x, y_viol) |>
  rowid_to_column()

draw_boot_samples <- function(dataset, resample_size, n_resamples){
  # dataset: dataframe or tibble
  # resample_size: integer, size of the sample to draw
  # n_resamples: integer, number of bootstrap samples to draw
  # returns list with each element a subset of dataset
  
  boot_accum <- list()
  for(i in 1:n_resamples){
    boot_accum[[i]] <- dataset |>
      slice_sample(n = resample_size, replace = TRUE)
  }
  return(boot_accum)
}


plot_boot_sample <- function(boot_sample){
  # boot_sample: dataset, outcome of draw_boot_sample
  boot_sample |>
    ggplot(aes(x = x, y = y_viol)) +
    geom_point() +
    geom_smooth(method = 'lm', se = FALSE) +
    xlim(-2, 2) +
    NULL
  
}

set.seed(1)
boot_samples <- draw_boot_samples(orig_data, boot_sample_size, n_resamples)
boot_plot_list <- lapply(boot_samples, plot_boot_sample)

# plot_boot_sample(boot_samples[[1]])
wrap_plots(boot_plot_list[1:12])

fit_lm <- function(boot_sample){
  # boot_sample: df with cols x, y_viol
  # returns coefs of LM fit to boot_sample
  
  m <- lm(y_viol ~ x, data = boot_sample)
  return(m$coefficients)
}

# fit_lm(boot_samples[[1]])
boot_coefs <- lapply(boot_samples, fit_lm)
boot_coefs_df <- boot_coefs |>
  bind_rows(.id = 'boot_idx')

boot_int_mean <- mean(boot_coefs_df$`(Intercept)`)
boot_int_sd <- sd(boot_coefs_df$`(Intercept)`)
boot_slp_mean <- mean(boot_coefs_df$x)
boot_slp_sd <- sd(boot_coefs_df$x)


p_intercept <- boot_coefs_df |>
  ggplot(aes(x = `(Intercept)`)) +
  geom_histogram(fill = 'grey') +
  geom_vline(xintercept = boot_int_mean, linewidth = 2, colour = 'black') +
  geom_vline(xintercept = boot_int_mean + boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dashed') +
  geom_vline(xintercept = boot_int_mean - boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dashed') + 
  geom_vline(xintercept = boot_int_mean + 1.96*boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
  geom_vline(xintercept = boot_int_mean - 1.96*boot_int_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
  geom_function(fun = function(x) dnorm(x, mean = boot_int_mean, sd = boot_int_sd) * 3, colour = 'black') +
  NULL

p_slope <- boot_coefs_df |>
  ggplot(aes(x = x)) +
  geom_histogram(fill = 'grey') +
  geom_vline(xintercept = boot_slp_mean, linewidth = 2, colour = 'black') +
  geom_vline(xintercept = boot_slp_mean + boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dashed') +
  geom_vline(xintercept = boot_slp_mean - boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dashed') + 
  geom_vline(xintercept = boot_slp_mean + 1.96*boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
  geom_vline(xintercept = boot_slp_mean - 1.96*boot_slp_sd, linewidth = 1, colour = 'black', linetype = 'dotted') +
  geom_function(fun = function(x) dnorm(x, mean = boot_slp_mean, sd = boot_slp_sd) * 3, colour = 'black') +
  NULL

p_intercept + p_slope

^ would be cool to animate the way these sampling distributions change as samples trickle in, the way I did for bayes_stat.

The big difficulty with bootstrapping is that to actually understand WHY it works, you need to understand the very very complicated maths behind it. So it’s not easy to give a good intuition for why it helps.

LS0tCnRpdGxlOiAiTGVjdHVyZSAwOSBwbGF5Z3JvdW5kIgpvdXRwdXQ6IAogIGh0bWxfbm90ZWJvb2s6CiAgICB0b2M6IHRydWUKLS0tCgpgYGB7cn0KbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkoY2FyKQpsaWJyYXJ5KHBlcmZvcm1hbmNlKQpsaWJyYXJ5KHBhdGNod29yaykKYGBgCgppZGVhOgoKLSBzaW0gZGF0YSB0aGF0IHZpb2xhdGVzIG5vcm1hbGl0eSBvZiBlcnJvcnMgYW5kIGhvbW9zY2VkLgotIGhhdmUgc3R1ZGVudHMgaWRlbnRpZnkgdGhvc2UgZXJyb3JzIGJhc2VkIG9uIGRpYWdub3N0aWMgcGxvdHMKLSBib290c3RyYXAgZGF0YSBmcm9tIHRoaXMgZGF0YXNldAotIHBsb3QgaXQgYW5kIHNlZSBpZiB0aG9zZSBhc3N1bXB0aW9ucyBjaGFuZ2UKCgojIFNpbSB2aW9sYXRpbmcgZGF0YQoKU29tZXRoaW5nIGRpZmZlcmVudCB3aXRoIHRoZSBlcnJvcnMgdGhpcyB0aW1lLgpHaXZlIHN0dWRlbnRzIG5ldyBpbGx1c3RyYXRpb24gb2YgaG93IHRoaW5ncyBjYW4gZGl2ZXJnZSBmcm9tIG5vcm1hbGl0eS4KTm90IGFzeW1tZXRyaWNhbCwgYnV0IG1heWJlIFUtc2hhcGVkIGVycm9ycz8gQXMgaW4sIG1vc3RseSBsYXJnZSBpbiBib3RoIGRpcmVjdGlvbnMsIG5vdCB1c3VhbGx5IHNtYWxsPwoKYGBge3J9CnNoYXBlcyA8LSAuMwpzZXQuc2VlZCgxKQpiZXRhX2Vycm9ycyA8LSByYmV0YSgxMDEsIHNoYXBlMSA9IHNoYXBlcywgc2hhcGUyID0gc2hhcGVzKSB8PgogIGRhdGF3aXphcmQ6OnN0YW5kYXJkaXplKCkKICAjIHNjYWxlKHNjYWxlID0gRkFMU0UsIGNlbnRlciA9IFRSVUUpCnBsb3QoYmV0YV9lcnJvcnMpCmhpc3QoYmV0YV9lcnJvcnMpCmBgYAoKYGBge3IgbWVzc2FnZSA9IEZ9CnNldC5zZWVkKDEpCgplcnJvcl9pbmNfdGVybXMgPC0gc2VxKC41LCAzLCBsZW5ndGgub3V0ID0gMTAxKQoKZGYgPC0gdGliYmxlKAogIHggPSBzZXEoLTIsIDIsIGxlbmd0aC5vdXQgPSAxMDEpLAogIHlfZ29vZCA9IDIgKyAoLTEuMSAqIHgpICsgcm5vcm0oMTAxLCAwLCAxKSwgCiAgeV92aW9sID0gMiArICgtMS4xICogeCkgKyBiZXRhX2Vycm9ycyAqIGVycm9yX2luY190ZXJtcywKKQoKcF9nb29kIDwtIGRmIHw+CiAgZ2dwbG90KGFlcyh4PXgsIHk9eV9nb29kKSkgKwogIGdlb21fcG9pbnQoKSArCiAgIyBnZW9tX3Ntb290aChtZXRob2QgPSAnbG0nLCBzZSA9IEZBTFNFKSArCiAgTlVMTAoKCnBfYmV0YSA8LSBkZiB8PgogIGdncGxvdChhZXMoeD14LCB5PXlfdmlvbCkpICsKICBnZW9tX3BvaW50KCkgKwogICMgZ2VvbV9zbW9vdGgobWV0aG9kID0gJ2xtJywgc2UgPSBGQUxTRSkgKwogIE5VTEwKCnBfZ29vZCArIHBfYmV0YQpgYGAKCiMjIEZpdCBtb2RlbHMKCmBgYHtyfQptX2dvb2QgPC0gbG0oeV9nb29kIH4geCwgZGF0YSA9IGRmKQptX3Zpb2wgPC0gbG0oeV92aW9sIH4geCwgZGF0YSA9IGRmKQpgYGAKCmBgYHtyfQpzdW1tYXJ5KG1fdmlvbCkKYGBgCgoKCiMjIFBsb3QgZGlhZ25vc3RpY3MKCmBgYHtyIGZpZy5oZWlnaHQgPSAxMH0KY2hlY2tfbW9kZWwobV92aW9sKQpgYGAKCgoKIyMjIFEtUSBwbG90CgpgYGB7cn0KcGxvdChtX3Zpb2wsIHdoaWNoID0gMikKYGBgCgoKIyMjIEhpc3RvZ3JhbSBvZiByZXNpZHVhbHMKCi0gYXN5bW1ldHJpY2FsCi0gZmF0dGVyIHRhaWxzIHRoYW4gd2Ugd291bGQgZXhwZWN0CgpgYGB7cn0KaGlzdChtX3Zpb2wkcmVzaWR1YWxzKQpgYGAKCkl0J3MgcmVhbGx5IGludGVyZXN0aW5nIGhvdyB0aGUgbW9kZWwgY2FuIGZpbmQgYSBsaW5lIHN1Y2ggdGhhdCB0aGUgcmVzaWR1YWxzIGxvb2sgYWxtb3N0IGdhdXNzaWFuLCBldmVuIHRob3VnaCB0aGUgZ2VuZXJhdGl2ZSBwcm9jZXNzIHdhcyBzbyBmYXIgZnJvbSBnYXVzc2lhbi4KCgojIyMgQ29tcGFyZSByZXNpZHVhbHMgdG8gZml0dGVkIHZhbHVlcwoKYGBge3J9CnBsb3QobV92aW9sLCB3aGljaCA9IDEpCmBgYAoKCiMgY2FyOjpCb290KCkKCiMjIGZpdCBtb2RlbAoKSW4gcmVhbGl0eSwgcHJlc2VudCB0aGlzIGFmdGVyIHNob3dpbmcgdGhlIG1hbnVhbCBwcm9jZXNzLgoKYEJvb3QoKWAgZHJhd3MgYFJgIHNhbXBsZXMgd2l0aCB0aGUgc2FtZSBudW1iZXIgb2Ygb2JzZXJ2YXRpb25zIGFzIHRoZSBvcmlnaW5hbCBkYXRhc2V0LgoKYGBge3J9CnNldC5zZWVkKDEpCm1fdmlvbF9ib290IDwtIEJvb3QobV92aW9sLCBSID0gMTAwMCkKc3VtbWFyeShtX3Zpb2xfYm9vdCkKYGBgClRoZSBTRSBvZiB0aGUgYm9vdHN0cmFwcGVkIHZlcnNpb24gaXMgc21hbGxlciAtLSB0aGVyZSBpcyBsZXNzIHZhcmlhYmlsaXR5IGluIHRoZSBib290c3RyYXBwZWQgZGF0YSB0aGFuIGluIHRoZSBvcmlnaW5hbCBzYW1wbGUuCkhlcmUncyB0aGUgU0Ugb2YgdGhlIG9yaWdpbmFsIHNhbXBsZToKCmBgYHtyfQpzdW1tYXJ5KG1fdmlvbCkkY29lZmZpY2llbnRzCmBgYAoKCmBgYHtyfQpDb25maW50KG1fdmlvbF9ib290LCBsZXZlbCA9IDAuOTUsIHR5cGUgPSAncGVyYycpCmBgYAoKQ29uc2VxdWVudGx5LCB0aGUgQ0lzIGFyZSBuYXJyb3dlciBmb3IgdGhlIGJvb3RzdHJhcHBlZCBtb2RlbCB0aGFuIHRoZXkgYXJlIGZvciB0aGUgb3JpZ2luYWwgbW9kZWwuCk9yaWdpbmFsIENJczoKCmBgYHtyfQpDb25maW50KG1fdmlvbCwgbGV2ZWwgPSAwLjk1LCB0eXBlID0gJ3BlcmMnKQpgYGAKCgojIyBwbG90IGRpYWdub3N0aWNzPwoKRG9lc24ndCBtYWtlIGEgdG9uIG9mIHNlbnNlIGJlY2F1c2UgZG9lc24ndCByZWFsbHkgcmVseSBvbiBzYW1lIGFzc3VtcHRpb25zIGFzIHRoZSBwYXJhbWV0cmljIHZlcnNpb24gb2YgdGhlIG1vZGVsLgoKYGBge3J9CnBsb3QobV92aW9sX2Jvb3QpCmBgYAoKSW50ZXJlc3RpbmcsIHNvIGFwcGFyZW50bHkgd2Ugd2FudCB0aGUgdC12YWx1ZXMgdG8gYmUgbm9ybWFsbHkgZGlzdHJpYnV0ZWQuCgpgY2hlY2tfbW9kZWwoKWAgdGhyb3dzIGFuIGVycm9yLgoKYGBge3J9CiMgY2hlY2tfbW9kZWwobV92aW9sX2Jvb3QpCmBgYAoKCgojIyBmYWlsOiBlbW1lYW5zIGRvZXNuJ3Qgd29yayBvbiBib290IG9iamVjdC4KCmBgYHtyfQojIG1fdmlvbF9lbW0gPC0gZW1tZWFucyggcmVmX2dyaWQobV92aW9sLCBhdCA9IGxpc3QoeCA9IGMoLTIsIC0xLCAwLCAxLCAyKSkpLCB+eCkKIyBtX3Zpb2xfZW1tIHw+IGFzLmRhdGEuZnJhbWUoKQpgYGAKCmBgYHtyfQojIG1fdmlvbF9ib290X2VtbSA8LSBlbW1lYW5zKCByZWZfZ3JpZChtX3Zpb2xfYm9vdCwgYXQgPSBsaXN0KHggPSBjKC0yLCAtMSwgMCwgMSwgMikpKSwgfngpCmBgYAoKCgoKCiMgTWFudWFsbHkgYm9vdHN0cmFwIGZyb20gdGhpcyBkYXRhCgotIGJvb3RzdHJhcCBzYW1wbGUKLSBmaXQgbW9kZWwgdG8gdGhhdCBkYXRhCi0gZ2F0aGVyIHZhbHVlcyBvZiBpbXBvcnRhbnQgY29lZnMgCi0gbWFrZSBkaXN0cmliIG9mIHRob3NlIGNvZWZzIAotIHN1bW1hcmlzZSB0aG9zZSBkaXN0cmlicwotIHRoYXQncyB3aGF0IHRoZSBtb2RlbCBpcyBkb2luZwoKYGBge3IgbWVzc2FnZSA9IEZ9CmJvb3Rfc2FtcGxlX3NpemUgPC0gMjUKbl9yZXNhbXBsZXMgPC0gNTAKCm9yaWdfZGF0YSA8LSBkZiB8PgogIHNlbGVjdCh4LCB5X3Zpb2wpIHw+CiAgcm93aWRfdG9fY29sdW1uKCkKCmRyYXdfYm9vdF9zYW1wbGVzIDwtIGZ1bmN0aW9uKGRhdGFzZXQsIHJlc2FtcGxlX3NpemUsIG5fcmVzYW1wbGVzKXsKICAjIGRhdGFzZXQ6IGRhdGFmcmFtZSBvciB0aWJibGUKICAjIHJlc2FtcGxlX3NpemU6IGludGVnZXIsIHNpemUgb2YgdGhlIHNhbXBsZSB0byBkcmF3CiAgIyBuX3Jlc2FtcGxlczogaW50ZWdlciwgbnVtYmVyIG9mIGJvb3RzdHJhcCBzYW1wbGVzIHRvIGRyYXcKICAjIHJldHVybnMgbGlzdCB3aXRoIGVhY2ggZWxlbWVudCBhIHN1YnNldCBvZiBkYXRhc2V0CiAgCiAgYm9vdF9hY2N1bSA8LSBsaXN0KCkKICBmb3IoaSBpbiAxOm5fcmVzYW1wbGVzKXsKICAgIGJvb3RfYWNjdW1bW2ldXSA8LSBkYXRhc2V0IHw+CiAgICAgIHNsaWNlX3NhbXBsZShuID0gcmVzYW1wbGVfc2l6ZSwgcmVwbGFjZSA9IFRSVUUpCiAgfQogIHJldHVybihib290X2FjY3VtKQp9CgoKcGxvdF9ib290X3NhbXBsZSA8LSBmdW5jdGlvbihib290X3NhbXBsZSl7CiAgIyBib290X3NhbXBsZTogZGF0YXNldCwgb3V0Y29tZSBvZiBkcmF3X2Jvb3Rfc2FtcGxlCiAgYm9vdF9zYW1wbGUgfD4KICAgIGdncGxvdChhZXMoeCA9IHgsIHkgPSB5X3Zpb2wpKSArCiAgICBnZW9tX3BvaW50KCkgKwogICAgZ2VvbV9zbW9vdGgobWV0aG9kID0gJ2xtJywgc2UgPSBGQUxTRSkgKwogICAgeGxpbSgtMiwgMikgKwogICAgTlVMTAogIAp9CgpzZXQuc2VlZCgxKQpib290X3NhbXBsZXMgPC0gZHJhd19ib290X3NhbXBsZXMob3JpZ19kYXRhLCBib290X3NhbXBsZV9zaXplLCBuX3Jlc2FtcGxlcykKYm9vdF9wbG90X2xpc3QgPC0gbGFwcGx5KGJvb3Rfc2FtcGxlcywgcGxvdF9ib290X3NhbXBsZSkKCiMgcGxvdF9ib290X3NhbXBsZShib290X3NhbXBsZXNbWzFdXSkKd3JhcF9wbG90cyhib290X3Bsb3RfbGlzdFsxOjEyXSkKYGBgCgoKYGBge3IgbWVzc2FnZT1GfQpmaXRfbG0gPC0gZnVuY3Rpb24oYm9vdF9zYW1wbGUpewogICMgYm9vdF9zYW1wbGU6IGRmIHdpdGggY29scyB4LCB5X3Zpb2wKICAjIHJldHVybnMgY29lZnMgb2YgTE0gZml0IHRvIGJvb3Rfc2FtcGxlCiAgCiAgbSA8LSBsbSh5X3Zpb2wgfiB4LCBkYXRhID0gYm9vdF9zYW1wbGUpCiAgcmV0dXJuKG0kY29lZmZpY2llbnRzKQp9CgojIGZpdF9sbShib290X3NhbXBsZXNbWzFdXSkKYm9vdF9jb2VmcyA8LSBsYXBwbHkoYm9vdF9zYW1wbGVzLCBmaXRfbG0pCmJvb3RfY29lZnNfZGYgPC0gYm9vdF9jb2VmcyB8PgogIGJpbmRfcm93cyguaWQgPSAnYm9vdF9pZHgnKQoKYm9vdF9pbnRfbWVhbiA8LSBtZWFuKGJvb3RfY29lZnNfZGYkYChJbnRlcmNlcHQpYCkKYm9vdF9pbnRfc2QgPC0gc2QoYm9vdF9jb2Vmc19kZiRgKEludGVyY2VwdClgKQpib290X3NscF9tZWFuIDwtIG1lYW4oYm9vdF9jb2Vmc19kZiR4KQpib290X3NscF9zZCA8LSBzZChib290X2NvZWZzX2RmJHgpCgoKcF9pbnRlcmNlcHQgPC0gYm9vdF9jb2Vmc19kZiB8PgogIGdncGxvdChhZXMoeCA9IGAoSW50ZXJjZXB0KWApKSArCiAgZ2VvbV9oaXN0b2dyYW0oZmlsbCA9ICdncmV5JykgKwogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IGJvb3RfaW50X21lYW4sIGxpbmV3aWR0aCA9IDIsIGNvbG91ciA9ICdibGFjaycpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X2ludF9tZWFuICsgYm9vdF9pbnRfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X2ludF9tZWFuIC0gYm9vdF9pbnRfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsgCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gYm9vdF9pbnRfbWVhbiArIDEuOTYqYm9vdF9pbnRfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2RvdHRlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X2ludF9tZWFuIC0gMS45Nipib290X2ludF9zZCwgbGluZXdpZHRoID0gMSwgY29sb3VyID0gJ2JsYWNrJywgbGluZXR5cGUgPSAnZG90dGVkJykgKwogIGdlb21fZnVuY3Rpb24oZnVuID0gZnVuY3Rpb24oeCkgZG5vcm0oeCwgbWVhbiA9IGJvb3RfaW50X21lYW4sIHNkID0gYm9vdF9pbnRfc2QpICogMywgY29sb3VyID0gJ2JsYWNrJykgKwogIE5VTEwKCnBfc2xvcGUgPC0gYm9vdF9jb2Vmc19kZiB8PgogIGdncGxvdChhZXMoeCA9IHgpKSArCiAgZ2VvbV9oaXN0b2dyYW0oZmlsbCA9ICdncmV5JykgKwogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IGJvb3Rfc2xwX21lYW4sIGxpbmV3aWR0aCA9IDIsIGNvbG91ciA9ICdibGFjaycpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X3NscF9tZWFuICsgYm9vdF9zbHBfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X3NscF9tZWFuIC0gYm9vdF9zbHBfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2Rhc2hlZCcpICsgCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gYm9vdF9zbHBfbWVhbiArIDEuOTYqYm9vdF9zbHBfc2QsIGxpbmV3aWR0aCA9IDEsIGNvbG91ciA9ICdibGFjaycsIGxpbmV0eXBlID0gJ2RvdHRlZCcpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSBib290X3NscF9tZWFuIC0gMS45Nipib290X3NscF9zZCwgbGluZXdpZHRoID0gMSwgY29sb3VyID0gJ2JsYWNrJywgbGluZXR5cGUgPSAnZG90dGVkJykgKwogIGdlb21fZnVuY3Rpb24oZnVuID0gZnVuY3Rpb24oeCkgZG5vcm0oeCwgbWVhbiA9IGJvb3Rfc2xwX21lYW4sIHNkID0gYm9vdF9zbHBfc2QpICogMywgY29sb3VyID0gJ2JsYWNrJykgKwogIE5VTEwKCnBfaW50ZXJjZXB0ICsgcF9zbG9wZQpgYGAKCgpeIHdvdWxkIGJlIGNvb2wgdG8gYW5pbWF0ZSB0aGUgd2F5IHRoZXNlIHNhbXBsaW5nIGRpc3RyaWJ1dGlvbnMgY2hhbmdlIGFzIHNhbXBsZXMgdHJpY2tsZSBpbiwgdGhlIHdheSBJIGRpZCBmb3IgYmF5ZXNfc3RhdC4KCgpUaGUgYmlnIGRpZmZpY3VsdHkgd2l0aCBib290c3RyYXBwaW5nIGlzIHRoYXQgdG8gYWN0dWFsbHkgdW5kZXJzdGFuZCBXSFkgaXQgd29ya3MsIHlvdSBuZWVkIHRvIHVuZGVyc3RhbmQgdGhlIHZlcnkgdmVyeSBjb21wbGljYXRlZCBtYXRocyBiZWhpbmQgaXQuClNvIGl0J3Mgbm90IGVhc3kgdG8gZ2l2ZSBhIGdvb2QgaW50dWl0aW9uIGZvciB3aHkgaXQgaGVscHMuCgoK